# coding=utf-8

import scrapy
from scrapy.loader import ItemLoader
from chinahr.items import JobInfoItem, ComInfoItem
from scrapyluke.processors import *
import json

__author__ = 'yanyan'


class Spider51job(scrapy.Spider):
    name = 'cjol'
    allowed_domains = ['cjol.com']
    start_urls = ['http://s.cjol.com/service/joblistjson.aspx?KeywordType=3&SearchType=2&ListType=2']

    def parse(self, response):
        data = json.loads(response.body_as_unicode())
        pages = (int(data['RecordSum'])-1)/40 + 1
        for i in range(1, pages + 1):
            yield scrapy.Request('%s&page=%d' % (response.url, i), callback=self.parse_urls)

    def parse_urls(self, response):
        data = json.loads(response.body_as_unicode())
        com_urls = re.findall('jobid=\"(\d+)\"', data["JobListHtml"])
        job_urls = re.findall('companyid=\"(\d+)\"', data["JobListHtml"])
        for url in job_urls:
            yield scrapy.Request('http://www.cjol.com/jobs/job-%s' % url, callback=self.parse_job_info)
        for url in com_urls:
            yield scrapy.Request('http://www.cjol.com/jobs/company-%s' % url, callback=self.parse_com_info)

    def parse_job_info(self, response):
        loader = ItemLoader(item=JobInfoItem(), response=response)
        loader.add_value('url', value=response.url)
        loader.add_xpath('job_name', u'//h1[@class="job_basicinfo_jobname"]/text()', ReplaceBlank(''))
        loader.add_xpath('job_recruNums', u'//div[@class="job_basicinfo_name"]/span/text()', re=u'(\d+)')
        loader.add_xpath('job_location', u'//span[@class="job_basicinfo_address"]/text()', ReplaceBlank(''))
        loader.add_xpath('job_update', u'//p[@class="job_basicinfo_date"]/text()[last()]', ReplaceBlank(''))
        loader.add_xpath('job_salary', u'//li[contains(text(), "月薪：")]/text()', re=u'月薪：(.*)')
        loader.add_xpath('job_experience', u'//li[contains(text(), "经验：")]/text()', re=u'经验：(.*)')
        loader.add_xpath('job_miniEdu', u'//li[contains(text(), "学历：")]/text()', re=u'学历：(.*)')
        loader.add_xpath('job_desc_req', u'//li[contains(text(), "专业：")]/text()', re=u'专业：(.*)')
        loader.add_xpath('job_reqLan', u'//li[contains(text(), "外语：")]/text()', re=u'外语：(.*)')
        loader.add_xpath('job_detail', u'//li[contains(text(), "计算机：")]/text()', re=u'计算机：(.*)')
        loader.add_xpath('job_nature', u'//li[contains(text(), "类型：")]/text()', re=u'类型：(.*)')
        loader.add_xpath('job_reqAge', u'//li[contains(text(), "年龄：")]/text()', re=u'年龄：(.*)')
        loader.add_xpath('job_reqSex', u'//li[contains(text(), "性别：")]/text()', re=u'性别：(.*)')
        loader.add_xpath('job_benefits', u'//li[contains(text(), "住宿：")]/text()', re=u'住宿：(.*)')
        loader.add_xpath('job_category', u'//p[contains(text(), "岗位分类：")]/../ul[1]/li/a/text()', ReplaceBlank(''), JoinL('|'))
        loader.add_xpath('job_desc', u'//div[@class="positionstatement clearfix"]//text()', ReplaceBlank(''), RemoveTagsL(), JoinL())
        return loader.load_item()

    def parse_com_info(self, response):
        loader = ItemLoader(item=ComInfoItem(), response=response)
        loader.add_value('url', value=response.url)
        loader.add_xpath('com_name', u'//h1[@class="companyinfoshow_name_box"]/text()', ReplaceBlank(''))
        loader.add_xpath('com_industry', u'//li[contains(text(), "行业：")]/text()', re=u'行业：(.*)')
        loader.add_xpath('com_address', u'//li[contains(text(), "地址：")]/text()', re=u'地址：(.*)')
        loader.add_xpath('com_size', u'//li[contains(text(), "公司规模：")]/text()', re=u'公司规模：(.*)')
        loader.add_xpath('com_nature', u'//li[contains(text(), "公司性质：")]/text()', re=u'公司性质：(.*)')
        loader.add_xpath('com_intro', u'//h2[contains(text(), "公司介绍")]/../div//text()', ReplaceBlank(''), RemoveTagsL(), JoinL(''))
        return loader.load_item()


